package com.lenovo.baidu.http;

import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.lenovo.util.WriteTxt;
import com.lenovo.redis.JedisPoolUtil;
import com.lenovo.redis.RedisName;

public class MainCrawler {
	
	
	 public static void main(String[] args)throws Exception {
		 int ik=50 ;
		 while(ik<=50){
			 
		
//		 Document doc = Jsoup.connect("http://en.wikipedia.org/").userAgent("Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20100101 Firefox/22.0").get();
//		 Elements newsHeadlines = doc.select("#mp-itn b a");
	        CloseableHttpClient httpclient = HttpClients.createDefault();
	        try {
	           // HttpHost target = new HttpHost("localhost", 443, "https");
	            HttpHost proxy = new HttpHost("10.4.10.190", 8080, "http");

	            RequestConfig config = RequestConfig.custom()
	                    .setProxy(proxy)
	                    .build();
	            HttpGet request = new HttpGet("http://tieba.baidu.com/f?kw=%B9%FA%B2%FA%CA%D6%BB%FA&pn="+ik);
	            
	            request.addHeader("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; "
						+ "Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; "
						+ "Media Center PC 6.0)");
				//request.addHeader("Host","tieba.baidu.com");
	            request.setConfig(config);

	            System.out.println("Executing request " + request.getRequestLine() + " to "+ " via " + proxy);

	            CloseableHttpResponse response = httpclient.execute(request);
	            try {
	            	System.out.println(response.getStatusLine());  
	                System.out.println("----------------------------------------");
	              //  System.out.println(response.getStatusLine());
	                HttpEntity entity=response.getEntity() ;
	                String content = EntityUtils.toString(entity);  
	                
	                Document doc = Jsoup.parse(content) ;

	                Elements links = doc.select("li.j_thread_list"); // a with href
	                //提取方式一:  
	              for(Element i:links)  
	              {  
	            	  StringBuffer buffer=new StringBuffer("");  //*****
	            	  String jsonUser=i.attr("data-field") ;
	            	  
	            	  Elements ele1=  i.getElementsByClass("tb_icon_author") ;
	            	  Elements eleren=  i.getElementsByClass("tb_icon_author_rely") ;
	            	  Elements ele2=  i.getElementsByClass("j_th_tit") ;
	            	 
	            	  JSONObject objUserID=  JSON.parseObject(ele1.get(0).attr("data-field"));
	            	  
	            	JSONObject objUser=  JSON.parseObject(jsonUser);
	            	  Elements reply_date = doc.getElementsByClass("threadlist_reply_date"); // a with href
	            	buffer.append(objUser.getString("id")) ;
	            	buffer.append("#!@lenovo!@#");
	            	buffer.append(i.text()) ;
	            	buffer.append("#!@lenovo!@#");
	            	buffer.append(ele1.get(0).attr("title")) ;
	            	buffer.append("#!@lenovo!@#");
	            	buffer.append(objUserID.getString("user_id")) ;
	            	buffer.append("#!@lenovo!@#");
	            	buffer.append(reply_date.get(0).text()) ;
	            	buffer.append("#!@lenovo!@#");
	            	buffer.append(eleren.get(0).attr("title")) ;
	            	buffer.append("#!@lenovo!@#");
	            	buffer.append(ele2.get(1).attr("href")) ;
	            	System.out.println(buffer.toString());
	            	
	            	  
  	                 
  	                
  	              //    System.out.println(	ele1.get(0).attr("data-field")+ele1.get(0).attr("title"));
	                //	System.out.println(i);
		                  String tag=i.tagName();  
		                  if(tag.equals("li")){  
		                	  	  
  		                       buffer.append("(i.text()") ;
		          //          System.out.println(i.text());  
		                     }
	                
	               }  
	          //     System.out.println(links.size()); 
	                  // img with src ending .png

	            
	                  // div with class=masthead

	              //  Elements resultLinks = doc.select("h3.r > a"); // direct a after h3
	               // System.out.println(response.getEntity());
	             //   System.out.println(content+"Response content:"  
	               //         );  
	            } finally {
	                response.close();
	            }
	        } finally {
	            httpclient.close();
	        }
	        ik+=50 ;
		 }
	    }
}
